{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#需要的包一股脑全放里"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import urllib2, urllib,sys,os\n",
    "import codecs,re,string,json\n",
    "from subprocess import *\n",
    "from collections import Counter\n",
    "from svmutil import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#分词标注部分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#分词与词性标注  以json格式输出\n",
    "def divPos(input):\n",
    "    uri_base = \"http://ltpapi.voicecloud.cn/analysis/?\"\n",
    "    api_key  = \"l3z7P327MjUbYOpUyR7m4kmobn7eVVrsaJlUSxlb\" \n",
    "    text= urllib.quote(input)\n",
    "    #语言云的真正调用方法    \n",
    "    uri_base = \"http://ltpapi.voicecloud.cn/analysis/?\"       \n",
    "    data = {\n",
    "            \"api_key\"   : \"l3z7P327MjUbYOpUyR7m4kmobn7eVVrsaJlUSxlb\",\n",
    "            \"text\"      : text,\n",
    "            \"format\"    : \"json\",\n",
    "            \"has_key\"   : \"false\",\n",
    "            \"pattern\"   : \"pos\",               \n",
    "\n",
    "            }\n",
    "    params = urllib.urlencode(data)\n",
    "    try:\n",
    "        request  = urllib2.Request(uri_base)\n",
    "        response = urllib2.urlopen(request, params)\n",
    "        content  = response.read().strip()\n",
    "        \n",
    "        return content \n",
    "    except urllib2.HTTPError, e:\n",
    "        print >> sys.stderr, e.reason\n",
    "#json格式转化成plain格式\n",
    "def jsonToPlain(contentJson):\n",
    "    #只保留分词和标注输出\n",
    "    #posResult=[word for word in sentence[0] for sentence in content for ]\n",
    "    resultList=[]\n",
    "    for paragraph in contentJson:\n",
    "        sentenceList=[]\n",
    "        if paragraph[0][-1][-1]== \"wp\":\n",
    "            for word in paragraph[0][:-1]:\n",
    "                if u\"转\" in word[1]:\n",
    "                    word[2]=\"v\"\n",
    "                sentenceList.append(word[1]+'\\t'+word[2])\n",
    "        else:\n",
    "            for word in paragraph[0]:\n",
    "                if u\"转\" in word[1]:\n",
    "                    word[2]=\"v\"\n",
    "                sentenceList.append(word[1]+'\\t'+word[2])\n",
    "        sentence='\\n'.join(sentenceList).encode('utf-8')\n",
    "        sentence1=re.sub( r'(nh|ni|nl|ns|nz)' , 'n' , sentence )\n",
    "        resultList.append(sentence1+'\\n'*2)\n",
    "\n",
    "    return resultList\n",
    "\n",
    "#分词调用程序\n",
    "def distinguishWord():\n",
    "    #测试句子，直接输入\n",
    "    dataa=raw_input('输入命令：'.decode('utf-8')).decode('gb2312').encode('utf-8')#程序中字符串为utf-8，交互中键盘敲入为gb2312\n",
    "    #输入句子，调用分词标注函数，输出结果（json）\n",
    "    if dataa=='':\n",
    "        return False\n",
    "    output=divPos(dataa)\n",
    "\n",
    "    #将分词标注结果输出到txt文档中\n",
    "    f1=open('distinguishWordJson','w')\n",
    "    f1.write(output)\n",
    "    f1.close()\n",
    "    \n",
    "    contentJson=json.loads(output)\n",
    "\n",
    "    resultList=jsonToPlain(contentJson)\n",
    "    f2=open('distinguishWordPlain','w')\n",
    "    f2.writelines(resultList)\n",
    "    f2.close()\n",
    "    \n",
    "    return True\n",
    "\n",
    "def train(template,trainTxt,model):      \n",
    "    #returnCode = call('crf_learn.exe template crfTrain1.txt model1')\n",
    "    call('crf_learn.exe '+template+' '+trainTxt+' '+ model)\n",
    "def test(model,testTxt,testOutput):\n",
    "    \"\"\"\n",
    "    -v1:输出标签的概率值\n",
    "    -n :输出几层不同概率只的选项 用处不大\n",
    "    -t:输出model的txt版本\n",
    "    -f：是一阀值，只有某词的频率大于该值 才有用\n",
    "    -c：跟拟合度有关的一个参数\n",
    "    -h: 可以随时打开帮助看看\n",
    "    \n",
    "    \"\"\"\n",
    "    #Popen('crf_test.exe -m model1 crfTest1.txt >output1.txt', shell = True, stdout = PIPE).stdout\n",
    "    Popen('crf_test.exe -m '+model+' '+ testTxt +'>'+testOutput, shell = True, stdout = PIPE).communicate()\n",
    "    #Popen('crf_test.exe -m '+model+' '+ testTxt +'>'+testOutput, shell = True, stdout = PIPE).communicate()\n",
    "def evaluate(testOutput,testResult):\n",
    "    \n",
    "    evalue=Popen('conlleval.pl -d \"\\t\" -r < '+testOutput+' > '+testResult,\n",
    "                 shell = True, stdout = PIPE).stdout\n",
    "    #https://argcv.com/articles/2104.c#respond   参考资料，参数使用很全\n",
    "    #print evalue.read()\n",
    "\n",
    "def semanticTrain():#http://www.hankcs.com/nlp/the-crf-model-format-description.html  不错的资料\n",
    "    train('semanticTemplate','semanticTrain','semanticModel')\n",
    "def semanticTest():\n",
    "    test('semanticModel','distinguishWordPlain','semanticOutput')\n",
    "    #test('model','crfTest1.txt','output1.txt')\n",
    "def semanticEvaluate():\n",
    "    evaluate('semanticEvaluate','semanticEvaluateOutput')\n",
    "\n",
    "#将普通的字符串格式的输入转化成json列表形式\n",
    "def plainToJson(semanticOutput):\n",
    "    fCrfTrain=open(semanticOutput,'r')\n",
    "    article=fCrfTrain.read()\n",
    "    if article[:3] == codecs.BOM_UTF8:\n",
    "            article = article[3:]  \n",
    "    fCrfTrain.close()\n",
    "\n",
    "    sentenceLabel=re.compile(r\"(.*?)\\n\\n\",re.S)\n",
    "    sentenceList=re.findall(sentenceLabel,article)\n",
    "\n",
    "    #print sentenceList[0:10]\n",
    "\n",
    "    wordListInSentenceList=[]\n",
    "    for sentence in sentenceList:\n",
    "        wordList=sentence.split('\\n')\n",
    "        wordPosYuyiList=[]\n",
    "        for word in wordList:\n",
    "            wordPosYuyi=word.split('\\t')\n",
    "            wordPosYuyiList.append(wordPosYuyi)\n",
    "        wordListInSentenceList.append(wordPosYuyiList)\n",
    "    return wordListInSentenceList\n",
    "#普通的字符串格式的输入（带有边缘概率需要处理）转化成json列表形式\n",
    "def proPlainToJson(semanticOutput):\n",
    "    fCrfTrain=open(semanticOutput,'r')\n",
    "    article=fCrfTrain.read()\n",
    "    if article[:3] == codecs.BOM_UTF8:\n",
    "            article = article[3:]  \n",
    "    fCrfTrain.close()\n",
    "\n",
    "    sentenceLabel=re.compile(r\"(.*?)\\n\\n\",re.S)\n",
    "    sentenceList=re.findall(sentenceLabel,article)\n",
    "\n",
    "    #print sentenceList[0:10]\n",
    "\n",
    "    wordListInSentenceList=[]\n",
    "    for sentence in sentenceList:\n",
    "        wordList=sentence.split('\\n')\n",
    "        \n",
    "        wordPosYuyiProList=[]\n",
    "        for word in wordList[1:]:\n",
    "            wordPosYuyiPro=word.split('\\t')\n",
    "            newWordPosYuyiPro=wordPosYuyiPro[:-1]+[wordPosYuyiPro[-1].split('/')[0]]\n",
    "            wordPosYuyiProList.append(newWordPosYuyiPro)\n",
    "        wordListInSentenceList.append([float(wordList[0][2:])]+wordPosYuyiProList)\n",
    "    return wordListInSentenceList\n",
    "#将名词和方向词合并 输出json格式\n",
    "def n_f(wordListInSentenceList):\n",
    "    for s in xrange(len(wordListInSentenceList)):\n",
    "        wordPosition=0\n",
    "        while wordPosition<len(wordListInSentenceList[s]):\n",
    "            nPosition=[]\n",
    "            while wordPosition<len(wordListInSentenceList[s]) and '7' in wordListInSentenceList[s][wordPosition][-1]:\n",
    "                \n",
    "                nPosition.append(wordPosition)\n",
    "                wordPosition+=1\n",
    "            #print nPosition \n",
    "            if len(nPosition)!=0:\n",
    "                #\n",
    "                words=[]\n",
    "                for p in nPosition:\n",
    "                    words.append(wordListInSentenceList[s][p][0])\n",
    "                word=''.join(words)\n",
    "                nTotal=[word,'n','7']\n",
    "                wordListInSentenceList[s]=wordListInSentenceList[s][:nPosition[0]]+[nTotal]+wordListInSentenceList[s][nPosition[-1]+1:]\n",
    "                #print wordListInSentenceList[s]\n",
    "                wordPosition=nPosition[0]\n",
    "                \n",
    "            vPosition=[]\n",
    "            while wordPosition<len(wordListInSentenceList[s]) and '-3' in wordListInSentenceList[s][wordPosition][-1]:\n",
    "                \n",
    "                vPosition.append(wordPosition)\n",
    "                wordPosition+=1\n",
    "            #print nPosition \n",
    "            if len(vPosition)!=0:\n",
    "                #\n",
    "                words=[]\n",
    "                for v in vPosition:\n",
    "                    words.append(wordListInSentenceList[s][v][0])\n",
    "                word=''.join(words)\n",
    "                vTotal=[word,'v','3']\n",
    "                wordListInSentenceList[s]=wordListInSentenceList[s][:vPosition[0]]+[vTotal]+wordListInSentenceList[s][vPosition[-1]+1:]\n",
    "                #print wordListInSentenceList[s]\n",
    "                wordPosition=vPosition[0]\n",
    "                \n",
    "            fPosition=[]\n",
    "            while wordPosition<len(wordListInSentenceList[s]) and '2' in wordListInSentenceList[s][wordPosition][-1]:\n",
    "                \n",
    "                fPosition.append(wordPosition)\n",
    "                wordPosition+=1\n",
    "            #print nPosition \n",
    "            if len(fPosition)!=0:\n",
    "                #\n",
    "                words=[]\n",
    "                for p in fPosition:\n",
    "                    words.append(wordListInSentenceList[s][p][0])\n",
    "                word=''.join(words)\n",
    "                fTotal=[word,'nd','2']\n",
    "                wordListInSentenceList[s]=wordListInSentenceList[s][:fPosition[0]]+[fTotal]+wordListInSentenceList[s][fPosition[-1]+1:]\n",
    "                #print wordListInSentenceList[s]\n",
    "                wordPosition=fPosition[0]\n",
    "            #print wordPosition\n",
    "            wordPosition+=1\n",
    "    return\n",
    "\n",
    "#将合并后的json格式转化成plain格式输出    \n",
    "def jsonToPlain1(contentJson):\n",
    "    #只保留分词和标注输出\n",
    "    #posResult=[word for word in sentence[0] for sentence in content for ]\n",
    "    resultList=[]\n",
    "    for paragraph in contentJson:\n",
    "        sentenceList=[]\n",
    "        for word in paragraph:\n",
    "            sentenceList.append(word[0]+'\\t'+word[1]+'\\t'+word[2])\n",
    "        sentence='\\n'.join(sentenceList)\n",
    "        resultList.append(sentence+'\\n'*2)\n",
    "    return resultList\n",
    "\n",
    "def merge():\n",
    "    wordListInSentenceList=plainToJson('semanticOutput')#crfTrain.txt  包含词  词性  语义 的plain文件\n",
    "    n_f(wordListInSentenceList)\n",
    "\n",
    "\n",
    "    data=json.dumps(wordListInSentenceList)\n",
    "    f3=open('n_f_outputTestJson','w')                #jsonData.txt对应crfTrain的合并结果\n",
    "    f3.writelines(data)\n",
    "    f3.close()\n",
    "\n",
    "    result=jsonToPlain1(wordListInSentenceList)\n",
    "    f4=open('n_f_outputTest','w')           #n_f.data对应jsonData.txt  plain格式\n",
    "    f4.writelines(result)\n",
    "    f4.close()\n",
    "    \n",
    "#所有句子的特征列表\n",
    "def sentProperty(listData):\n",
    "    property_all=[]\n",
    "    for sentence in listData:\n",
    "        #句子中的语义列表\n",
    "        yuyiList=[ word[-1] for word in sentence]\n",
    "        yuyiStr=' '+' '.join(yuyiList)\n",
    "        #除0外的词占句子的比例\n",
    "        property1=1-Counter(yuyiList)['0']/len(yuyiList)\n",
    "        #是否有动词\n",
    "        property2=1 if re.search(r'3_|4',yuyiStr) else 0\n",
    "        \n",
    "        #是否含有方向词\n",
    "        property3=1 if '2' in yuyiList else 0\n",
    "        #是否含有地名词\n",
    "        property4=1 if '7' in yuyiList else 0\n",
    "        #是否含有数词加量词的集合\n",
    "        property5=1 if  re.search(r'6 8',yuyiStr)  else 0\n",
    "        #方向词加动词的个数\n",
    "        property6=len(re.findall(r'2 [34]',yuyiStr))\n",
    "        #动词加地名词的个数\n",
    "        property7=len(re.findall(r'3_? 7| 4 7',yuyiStr))\n",
    "        #地名词加动词\n",
    "        property8=len(re.findall(r'7 [34]',yuyiStr))\n",
    "        #动词的个数\n",
    "        property9=len(re.findall(r'3_| 4',yuyiStr))\n",
    "        #关键词类别个数占句子长度的比例\n",
    "        property10=len(set(yuyiList))/len(yuyiList)\n",
    "        \n",
    "        property_all.append([property1,property2,property3,property4,property5,property6,property7,property8,property9,property10])\n",
    "    return property_all\n",
    "\n",
    "#给每个句子贴上1或-1的标签，自己标出来\n",
    "def label_initialize():\n",
    "    set_label_1=[1]*60\n",
    "    label_0=[7,53,54,55,56,57,58,59]\n",
    "    for label in label_0:\n",
    "        set_label_1[label]=-1\n",
    "    return set_label_1\n",
    "\n",
    "#将句子的特征和1、-1标签弄成svm的输入格式\n",
    "def svm_input(property_all,label_tag):\n",
    "    svm_format=[]\n",
    "    for index in xrange(len(property_all)):               \n",
    "        column=str(label_tag[index])\n",
    "        for property_index in xrange(len(property_all[index])):\n",
    "            column+='\\t'+str(property_index+1)+':'+str(property_all[index][property_index])   \n",
    "        svm_format.append(column+'\\n')\n",
    "    return svm_format\n",
    "\n",
    "def svmProperty(n_f_input):\n",
    "#从合并后的带有语义标注的预料中  提取出svm的特征向量  以及自己标注的标签 存放在一个文本文档中\n",
    "    f5=open(n_f_input,'r')  #jsonData\n",
    "    jsonData1=f5.read()\n",
    "    f5.close()\n",
    "    listData=json.loads(jsonData1)\n",
    "    sentPropertyAll=sentProperty(listData)\n",
    "\n",
    "    #sentLabel=label_initialize()\n",
    "    #sentLabel=[0]*len(sentPropertyAll)\n",
    "    \n",
    "    return sentPropertyAll\n",
    "\n",
    "#训练模型  \n",
    "def svmTrain():\n",
    "    sentPropertyAll=svmProperty('jsonData')\n",
    "    sentLabel=label_initialize()\n",
    "    m=svm_train(sentLabel,sentPropertyAll,'-c 4')\n",
    "    svm_save_model('svmModel', m)\n",
    "    \n",
    "#进行测试\n",
    "def svmTest():\n",
    "    #y,x=svm_read_problem('svm_input_test.txt')\n",
    "    x=svmProperty('n_f_outputTestJson')\n",
    "    y=[0]*len(x)\n",
    "    m = svm_load_model('svmModel')\n",
    "    p_label,p_acc,p_val=svm_predict(y,x,m)\n",
    "    return p_label\n",
    "\n",
    "def splitTrain():\n",
    "    train('splitTemplate','splitTrain','splitModel')\n",
    "\n",
    "def splitTest():\n",
    "    test('splitModel','n_f_outputTest','splitOutput')\n",
    "def startFinishTrain():\n",
    "    train('startFinishTemplate','startFinishTrain','startFinishmodel')\n",
    "def startFinishTest():\n",
    "    test('startFinishmodel','splitOutput','startFinishOutput')\n",
    "def extract(output):\n",
    "    wordList=plainToJson(output)[0]\n",
    "    splitWordList=[]\n",
    "    beginLabel=[]\n",
    "    for i in xrange(len(wordList)):\n",
    "        if 'h' in wordList[i][3]:\n",
    "            beginLabel.append(i)\n",
    "    beginLabel.append(len(wordList))\n",
    "    \n",
    "    for i in xrange(len(beginLabel)-1):\n",
    "        splitWordList.append(wordList[beginLabel[i]:beginLabel[i+1]])\n",
    "    #print splitWordList\n",
    "    structureList=[]\n",
    "    for unit in splitWordList:\n",
    "        structure=['__']*6\n",
    "        for i in xrange(len(unit)):\n",
    "            if unit[i][-1]=='s': \n",
    "                structure[0]=unit[i][0]\n",
    "            elif unit[i][-1]=='f':\n",
    "                structure[1]=unit[i][0]\n",
    "            elif '3' in unit[i][2]:\n",
    "                structure[2]=unit[i][0]\n",
    "            elif '4' in unit[i][2]:\n",
    "                structure[2]=unit[i][0]\n",
    "            elif unit[i][2]=='2':\n",
    "                structure[3]=unit[i][0]\n",
    "            elif unit[i][2]=='6' and unit[i+1][2]=='8':            \n",
    "                structure[4]=unit[i][0]+unit[i+1][0]  \n",
    "            elif unit[i][2]=='9':\n",
    "                structure[4]=unit[i][0]\n",
    "            elif unit[i][2]=='5':\n",
    "                structure[5]=unit[i][0]\n",
    "            elif unit[i][2]=='6' and unit[i+1][2]=='11':            \n",
    "                structure[5]=unit[i][0]+unit[i+1][0]  \n",
    "        structureList.append(structure)\n",
    "    return structureList\n",
    "def testAll():\n",
    "    print \"输出格式为： 动作过程（起点，终点，动作，方向，距离，速度）\"\n",
    "    while distinguishWord():\n",
    "        semanticTest()\n",
    "        merge()\n",
    "        \n",
    "        if svmTest()>0:\n",
    "            \n",
    "            splitTest()\n",
    "            startFinishTest()\n",
    "            \n",
    "            structureList=extract('startFinishOutput')\n",
    "            #print structureList\n",
    "            for i in range(len(structureList)):\n",
    "                if \"停\" in structureList[i][2] or \"头\" in structureList[i][2]:\n",
    "                    print \"\\t动作过程（\"+','.join(structureList[i])+\"）\"\n",
    "                    continue\n",
    "                \n",
    "                if structureList[i][0]=='__':\n",
    "                    if i != 0 and structureList[i-1][1] !='__':\n",
    "                        structureList[i][0]=structureList[i-1][1]\n",
    "                    else:\n",
    "                        structureList[i][0]='当下位置'\n",
    "                if structureList[i][1]=='__' and structureList[i][4]=='__':\n",
    "                    if i !=len(structureList)-1 and structureList[i+1][0] !='__':\n",
    "                        structureList[i][1]=structureList[i+1][0]\n",
    "                if structureList[i][2]=='__':\n",
    "                    if i !=0:\n",
    "                        structureList[i][2]=structureList[i-1][2]\n",
    "                    else:\n",
    "                        structureList[i][2]='走'\n",
    "                if structureList[i][3]=='__' and structureList[i][1]=='__':\n",
    "                       \n",
    "                    if \"后\" in structureList[i][2]:\n",
    "                        structureList[i][3]='后'\n",
    "                    elif \"左\" in structureList[i][2]:\n",
    "                        structureList[i][3]='左'\n",
    "                    elif \"右\" in structureList[i][2]:\n",
    "                        structureList[i][3]='右'\n",
    "                    else:\n",
    "                        structureList[i][3]='前'\n",
    "                if structureList[i][5]=='__':\n",
    "                    structureList[i][5]='默认速度'\n",
    "                print \"\\t动作过程（\"+','.join(structureList[i])+\"）\"\n",
    "        #print \"\\n\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "输出格式为： 动作过程（起点，终点，动作，方向，距离，速度）\n",
      "输入命令：朝着大街方向直行进入民权门桥。\n",
      "Accuracy = 0% (0/1) (classification)\n",
      "\t动作过程（当下位置,__,直行,大街方向,__,默认速度）\n",
      "\t动作过程（当下位置,民权门桥,进入,__,__,默认速度）\n",
      "输入命令：继续沿石安高速公路行驶4.6公里。\n",
      "Accuracy = 0% (0/1) (classification)\n",
      "\t动作过程（石安高速公路,__,行驶,前,4.6公里,默认速度）\n",
      "输入命令：在解放桥左前方转弯进入五一大街。\n",
      "Accuracy = 0% (0/1) (classification)\n",
      "\t动作过程（解放桥,__,转弯,左前方,__,默认速度）\n",
      "\t动作过程（当下位置,五一大街,进入,__,__,默认速度）\n",
      "输入命令：\n"
     ]
    }
   ],
   "source": [
    "if __name__==\"__main__\":\n",
    "    testAll()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sen=\"\"\"沿\tp\n",
    "东\tnd\n",
    "长安街\tns\n",
    "行驶\tv\n",
    "940\tm\n",
    "米\tq\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "沿\tp\n",
      "东\tnd\n",
      "长安街\tns\n",
      "行驶\tv\n",
      "940\tm\n",
      "米\tq\n"
     ]
    }
   ],
   "source": [
    "p = re.compile( '(nh|ni|nl|ns|nz)')\n",
    "p.sub( 'n', sen)\n",
    "print sen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "沿\tp\n",
      "东\tnd\n",
      "长安街\tn\n",
      "行驶\tv\n",
      "940\tm\n",
      "米\tq\n"
     ]
    }
   ],
   "source": [
    "print re.sub( r'(nh|ni|nl|ns|nz)' , 'n' , sen )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
